import json

import requests
from flask import request, g, jsonify, current_app, stream_with_context, Response

from apps.apis import api
from apps.core.auth import verify_json_schema, verify_token_auth
from apps.core.baserequest import xml, HEADER, url_once_pull
from apps.core.database import db
from apps.core.mwebhdfs import change_hdfs
from apps.core.schemaall import token_schema, get_sub_db_field_schema, \
    oozie_log_schema, get_rule_list_schema, get_rule_log_schema, get_file_list_schema, extract_rule_schema, \
    oozie_stat_schema
from apps.logger.web_logger import weblog
from apps.models import Rule, SubDb, SourceType, GatherField, LogUser
from apps.utils.return_info import ReturnInfo


@api.route("/pulldata/get_file_list", methods=['POST'])
@verify_json_schema(get_file_list_schema)
@verify_token_auth(allowtokennull=False)
def get_file_list():
    """
    通过rule 规则的id 获取运行结果的目录和文件列表
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: token.
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            ruleid:
              type: integer
              description: 规则列表返回的id
              example: 7
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json
    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"文件移动成功","data":[]}
    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    ruleid = jsons.get("ruleid")
    rule = Rule.query.filter_by(id=ruleid).first()
    if rule is None:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "规则id 不存在"
        return jsonify(ret_obj.todict())
    if rule.rule_type == 0:
        pull_dir = current_app.config.get("PULL_ONCE")
    elif rule.rule_type == 1:
        pull_dir = current_app.config.get("PULL_ROUTINE")
    else:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "数据库　rule_type　错误"
        return jsonify(ret_obj.todict())
    pull_dir = pull_dir + "/" + str(rule.userid) + "_" + str(rule.id)
    web_hdfs_uri = current_app.config.get("WEB_HDFS")

    kwargs = {}
    kwargs["user.name"] = current_app.config.get("FILES_USER")
    kwargs["overwrite"] = False
    requests_session = requests.api

    response = ""
    try:
        kwargs["op"] = "LISTSTATUS"
        response = requests_session.request("get",
                                            "{}{}".format(web_hdfs_uri, pull_dir),
                                            params=kwargs,
                                            timeout=30,
                                            allow_redirects=False)
        if change_hdfs(response):
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "webhdfs 已切换,请重新访问"
            ret_obj.data = {}
            return jsonify(ret_obj.todict())
        lists = []
        for item in response.json()["FileStatuses"]["FileStatus"]:
            dicts = {}
            dicts["length"] = item["length"]
            dicts["filename"] = item["pathSuffix"]
            lists.append(dicts)
        ret_obj.data = lists

    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "查询{}目录错误".format("{}{}".format(web_hdfs_uri, pull_dir))
        ret_obj.data = {"errmsg": str(e), "result": response.json()}
        return jsonify(ret_obj.todict())

    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "获取文件列表成功"
    return jsonify(ret_obj.todict())


@api.route("/pulldata/oozie_log", methods=['POST'])
@verify_json_schema(oozie_log_schema)
@verify_token_auth(allowtokennull=False, username="app")
def oozie_log():
    """
    提取日志回写接口
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            userid:
              type: integer
              description: 用户的自增id
              example: 1
            username:
              type: string
              description: 用户名
              example: 'admin'
            ip:
              type: string
              description: 用户ip
              example: '192.168.31.121'
            ruleid:
              type: string
              description: 规则的自增id
              example: '1'
            file_type:
              type: integer
              description: 0为题录和引文都有,1为题录，2为引文
              example: 0
            log_json:
              type: object
              description: 日志信息的字典
              example: {"a":"b","c":"d"}
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: string
            description: 无
    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":“”}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json

    userid = jsons.get("userid")
    username = jsons.get("username")
    ip = jsons.get("ip")
    ruleid = jsons.get("ruleid")
    file_type = jsons.get("file_type")
    log_json = jsons.get("log_json")

    loguser = LogUser(userid=userid,
                      username=username,
                      ip=ip,
                      ruleid=ruleid,
                      file_type=file_type,
                      log_type=1,
                      log_json=log_json)

    try:
        db.session.add(loguser)
        db.session.commit()
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "loguser 写入数据库失败"
        ret_obj.data = {"errmsg": str(e), "results": jsons}
        return jsonify(ret_obj.todict())

    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "写入数据库成功"
    return jsonify(ret_obj.todict())


@api.route("/pulldata/oozie_stat", methods=['POST'])
@verify_json_schema(oozie_stat_schema)
@verify_token_auth(allowtokennull=False, username="app")
def oozie_stat():
    """
    提取状态回写接口
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            ruleid:
              type: integer
              description: 规则的自增id
              example: 1
            stat:
              type: string
              description: '已提交: SUBMITTED; 运行中: RUNNING; 已完成: FINISHED; 已失败: FAILED; 已被杀: KILLED。'
              example: 'RUNNING'
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: string
            description: 无
    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":“”}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    ruleid = jsons.get("ruleid")
    stat = jsons.get("stat")

    rule = Rule.query.filter_by(id=ruleid).first()
    if rule is None:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "规则id 不存在"
        return jsonify(ret_obj.todict())
    rule.stat = stat
    try:
        db.session.add(rule)
        db.session.commit()
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "loguser 写入数据库失败"
        ret_obj.data = {"errmsg": str(e), "results": jsons}
        return jsonify(ret_obj.todict())

    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "写入数据库成功"
    return jsonify(ret_obj.todict())


@api.route("/pulldata/get_sub_db_field", methods=['POST'])
@verify_json_schema(get_sub_db_field_schema)
@verify_token_auth(allowtokennull=False)
def get_sub_db_type():
    """
    查询子库的字段及说明
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            sub_db_id:
              type: string
              description: 五位的sub_db_id
              example: '00001'
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":{"result": {"字段1":"说明1","字段2":"说明2"}}}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    sub_db_id = jsons.get("sub_db_id")
    try:
        subdb = SubDb.query.filter_by(sub_db_id=sub_db_id).first()
        source_type = subdb.source_type
        sourcetype = SourceType.query.filter_by(type_id=source_type).first()
        listname1 = sourcetype.type_name_alias.split(";")
        listname1.append(sourcetype.type_name)
        listname2 = [r for r in listname1 if r != ""]
        gatherfieldall = GatherField.query.filter(GatherField.doc_type.in_(listname2),
                                                  GatherField.field_intro != '-1').all()
        # gatherfieldall = GatherField.query.filter(GatherField.doc_type == sourcetype.type_name,
        #                                           GatherField.field_intro != '-1').all()
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "查询数据库失败"
        ret_obj.data = {"errmsg": str(e), "results": jsons}
        return jsonify(ret_obj.todict())
    dicts = {}
    for gatherfield in gatherfieldall:
        dicts[gatherfield.field_name] = gatherfield.field_intro
    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "返回字段的字典"
    ret_obj.data = {"result": dicts}
    return jsonify(ret_obj.todict())


@api.route("/pulldata/get_ref_field", methods=['POST'])
@verify_json_schema(token_schema)
@verify_token_auth(allowtokennull=False)
def get_ref_field():
    """
    查询引文的字段及说明
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg

    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":{"result": {"字段1":"说明1","字段2":"说明2"}}}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    try:
        gatherfieldall = GatherField.query.filter(GatherField.doc_type == "参考文献",
                                                  GatherField.field_intro != '-1').all()

    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "查询数据库失败"
        ret_obj.data = {"errmsg": str(e), "results": jsons}
        return jsonify(ret_obj.todict())
    dicts = {}
    for gatherfield in gatherfieldall:
        dicts[gatherfield.field_name] = gatherfield.field_intro
    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "返回字段的字典"
    ret_obj.data = {"result": dicts}
    return jsonify(ret_obj.todict())


@api.route("/pulldata/get_rule_list", methods=['POST'])
@verify_json_schema(get_rule_list_schema)
@verify_token_auth(allowtokennull=False)
def get_rule_list():
    """
    查看自己建立的提取规则列表
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            rule_type:
              type: integer
              description: 0为单次提取规则，1为常规提取规则,2为提取全部
              example: 2
            page:
              type: integer
              description: 页码
              example: 1
            size:
              type: integer
              description: 分页大小,最大后台限制 100条数据 默认 10
              example: 10

    definitions:
      RuleTable:
        type: object
        properties:
          id:
            type: integer
            description: 规则的id, 查询日志和查看运行结果时需要
            example: 1
          oozie_id:
            type: string
            description: 查询规则运行状态id
            example: 0000075-191012235641226-oozie-vipc-W
          rule_type:
            type: integer
            description: 0为单次提取规则，1为常规提取规则
            example: 0
          rule:
            type: object
            description: 规则字典
          create_time:
            type: string
            description: 规则创建时间
            example: '2020-02-23 16:01:01'
      ReturnInfoRuleList:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
            properties:
              result:
                type: object
                list_rule:
                  type: array
                  items:
                    $ref: '#/definitions/RuleTable'
                total:
                  type: integer
                  description: 返回的信息全部在data中获取
                  example: 107


    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfoRuleList'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":{"result": {"list_rule": [{"id": "查询日志使用","oozie_id": "运行状态id","rule": "规则列表","rule_type": "规则类型","create_time":"创建时间","is_deprecated":"0为正常１为废弃"}],"total": "数据总量","stat":"运行状态"}}}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    rule_type = jsons.get("rule_type")
    page = jsons.get("page")
    size = jsons.get("size")
    if rule_type == 2:
        pagination = Rule.query.filter_by(username=g.user.username) \
            .order_by(Rule.update_time.desc()).paginate(page, per_page=size, error_out=False)

    else:
        pagination = Rule.query.filter_by(username=g.user.username, rule_type=rule_type) \
            .order_by(Rule.update_time.desc()).paginate(page, per_page=size, error_out=False)
    list_rule = []
    for item in pagination.items:
        dicts = {}
        rule = item.rule
        id_ = item.id
        rule_type = item.rule_type
        oozie_id = item.oozie_id
        userid = item.userid
        is_deprecated = item.is_deprecated

        dicts["rule"] = rule
        dicts["id"] = id_
        dicts["rule_type"] = rule_type
        dicts["oozie_id"] = oozie_id
        dicts["rule_id"] = str(userid) + "_" + str(id_)
        dicts["create_time"] = item.create_time.strftime('%Y-%m-%d %H:%M:%S')
        dicts["is_deprecated"] = is_deprecated
        dicts["stat"] = item.stat

        list_rule.append(dicts)

    result = {"total": pagination.total, "list_rule": list_rule}
    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "查询日志成功"
    ret_obj.data = {"result": result}
    return jsonify(ret_obj.todict())


@api.route("/pulldata/get_rule_log", methods=['POST'])
@verify_json_schema(get_rule_log_schema)
@verify_token_auth(allowtokennull=False)
def get_rule_log():
    """
    通过规则id 获取这条规则的日志
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: 用户认证. 为空被当作游客,将不认证 空为 "" (空字符串)
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            rule_id:
              type: integer
              description: 规则id
              example: 1
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json

    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"成功","data":[{"log_json":{"aa":"bb"},"create_time":"创建时间"}]}

    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    rule_id = jsons.get("rule_id")
    loguserAll = LogUser.query.filter_by(ruleid=rule_id).order_by(LogUser.update_time.desc()).all()
    list_log = []
    for loguser in loguserAll:
        list_log.append(
            {"log_json": loguser.log_json, "create_time": loguser.create_time.strftime('%Y-%m-%d %H:%M:%S')})
    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "查询日志成功"
    ret_obj.data = list_log
    return jsonify(ret_obj.todict())


@api.route("/pulldata/extract_rule", methods=['POST'])
@verify_json_schema(extract_rule_schema)
@verify_token_auth(allowtokennull=False)
def extract_rule():
    """
    单次提取和常规提取规则
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: token.
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            rule:
              type: array
              description: 规则列表
              items:
                $ref: '#/definitions/Rule'
            rule_type:
              type: integer
              description: 单次提取0或者常规提取1
              example: 0
    definitions:
      Rule:
        type: object
        properties:
          sub_db_id:
            type: string
            description: 子库id
            example: '00001'
          type:
            type: string
            description: meta or ref
            example: meta
          pub_year:
            type: string
            description: 年份段
            example: 2010-2020
          field:
            type: object
            description: 字段
            example: {"title":"标题", "author":"作者"}
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json
    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"文件移动成功","data":{"oozie_id":"查看单次提取的状态","rule_id":"规则表的自增id"}}
    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    rule_feild = jsons.get("rule")
    rule_type = jsons.get("rule_type")
    metafeilds = False
    reffeilds = False
    for feilds in rule_feild:
        if feilds["type"] == "meta":
            metafeilds = True
        if feilds["type"] == "ref":
            reffeilds = True
    if metafeilds and reffeilds:
        file_type = 0
    elif metafeilds:
        file_type = 1
    elif reffeilds:
        file_type = 2
    else:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "rule 字段中没有引文或者题录"
        return jsonify(ret_obj.todict())

    rule = Rule(username=g.user.username,
                userid=g.user.userid,
                rule_type=rule_type,
                oozie_id="",
                rule=rule_feild,
                stat='SUBMITTED')
    try:
        db.session.add(rule)
        db.session.commit()
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "写入数据库错误"
        ret_obj.data = {"errmsg": str(e)}
        return jsonify(ret_obj.todict())
    # 创建目录
    web_hdfs_uri = current_app.config.get("WEB_HDFS")
    PULL_ONCE = current_app.config.get("PULL_ONCE")
    PULL_ROUTINE = current_app.config.get("PULL_ROUTINE")
    dirname = str(g.user.userid) + "_" + str(rule.id)
    filename = str(g.user.userid) + "_" + str(rule.id) + ".rule"

    dicArgs = {
        "logdicts": {
            "file_type": file_type,
            "userid": int(g.user.userid),
            "username": g.user.username,
            "ip": request.remote_addr,
            "ruleid": str(rule.id)
        },
        "rule_id": str(g.user.userid) + "_" + str(rule.id),
        "extract_count": 0,
        "rule_type": rule_type,
        "rule": rule_feild
    }

    if rule_type == 0:
        dirpath = "{}{}/{}".format(web_hdfs_uri, PULL_ONCE, dirname)
        filepath = "{}{}/{}/{}".format(web_hdfs_uri, PULL_ONCE, dirname, filename)
    else:
        dirpath = "{}{}/{}".format(web_hdfs_uri, PULL_ROUTINE, dirname)
        filepath = "{}{}/{}/{}".format(web_hdfs_uri, PULL_ROUTINE, dirname, filename)
    kwargs = {}
    kwargs["user.name"] = "app"
    kwargs["op"] = "MKDIRS"
    kwargs["permission"] = 775

    requests_session = requests.api
    # 创建目录
    response = requests_session.request("put",
                                        dirpath,
                                        params=kwargs,
                                        timeout=30,
                                        allow_redirects=False)
    if change_hdfs(response):
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "webhdfs 已切换,请重新访问"
        ret_obj.data = {}
        return jsonify(ret_obj.todict())
    if response.json()["boolean"] != True:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "创建目录{}失败".format(dirpath)
        return jsonify(ret_obj.todict())
    # 写入rule 文件
    kwargs["op"] = "CREATE"
    kwargs["overwrite"] = False
    response = requests_session.request("put",
                                        filepath,
                                        params=kwargs,
                                        timeout=30,
                                        allow_redirects=False)
    # 正常状态码 307
    if response.status_code == 307:
        response = requests_session.put(response.headers['location'],
                                        data=json.dumps(dicArgs, ensure_ascii=False).encode('utf-8'))
        # 第二次请求正常状态码 201
        if response.status_code != 201:
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "创建文件{}，第二步请求失败".format(filepath)
            ret_obj.data = {"result": "状态码:{}".format(response.status_code)}
            return jsonify(ret_obj.todict())
    else:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "创建文件{}，第一步请求失败".format(filepath)
        ret_obj.data = {"result": "状态码:{}".format(response.status_code)}
        return jsonify(ret_obj.todict())
    oozie_id = ''
    if rule_type == 0:
        r = None
        r = requests.post(
            url_once_pull, data=xml % json.dumps(dicArgs), headers=HEADER)
        try:
            oozie_id = json.loads(r.text, encoding='utf8')["id"]
            rule.oozie_id = oozie_id
            try:
                db.session.add(rule)
                db.session.commit()
            except Exception as e:
                ret_obj.msg_code = "FAILED"
                ret_obj.msg = "oozie_id 写入数据库失败"
                ret_obj.data = {"errmsg": str(e), "results": jsons}
                return jsonify(ret_obj.todict())
        except Exception as e:
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "请求oozie失败"
            if r is not None:
                ret_obj.data = {"errmsg": str(e), "result": {"oozieresult": r.text, "request": jsons}}
            else:
                ret_obj.data = {"errmsg": str(e), "results": jsons}
            return jsonify(ret_obj.todict())
    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "请求成功"
    ret_obj.data = {"oozie_id": oozie_id, "rule_id": rule.id}
    return jsonify(ret_obj.todict())


@api.route("/pulldata/get_rule_status", methods=['POST'])
@verify_json_schema(get_file_list_schema)
@verify_token_auth(allowtokennull=False)
def get_rule_status():
    """
    获取提取的状态
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: token.
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            ruleid:
              type: integer
              description: 规则id
              example: 1
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json
    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"文件移动成功","data":{"status": "运行的状态"}}
    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    ruleid = jsons.get("ruleid")
    rule = Rule.query.filter_by(id=ruleid).first()
    if rule is None:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "规则id 不存在"
        return jsonify(ret_obj.todict())

    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "请求成功"
    ret_obj.data = {"status": rule.stat}
    return jsonify(ret_obj.todict())


@api.route("/pulldata/routine_rule_deprecated", methods=['POST'])
@verify_json_schema(get_file_list_schema)
@verify_token_auth(allowtokennull=False)
def routine_rule_deprecated():
    """
    废弃某条常规规则
    ---
    tags:
      - pulldata
    parameters:
      - in: body
        name: body
        required: true
        description: JSON 参数.
        schema:
          properties:
            token:
              type: string
              description: token.
              example: eyJhbGciOiJIUzUxMiIsImlhdCI6MTU3Mjg0NzY3OSwiZXhwIjoxNTcyODQ4Mjc5fQ.eyJpZCI6MX0.rLbJ_6UQHuHoEA2xjbqtUa28Ogtf2I4JEDa0En0WjgB5pe5L1gT6F5L0RktZuJBE1lBEGT0TAJ1YC7PddwE-Cg
            ruleid:
              type: integer
              description: 规则列表返回的id
              example: 7
    definitions:
      ReturnInfo:
        type: object
        properties:
          msg_code:
            type: string
            description: SUCCESS 或者 FAILED.
            example: SUCCESS
          msg:
            type: string
            description: 说明文字，帮助找到失败原因
            example: 不能使用get请求
          data:
            type: object
            description: 返回的信息全部在data中获取
    consumes:
      - application/json
    produces:
      - application/json
    responses:
      200:
        description: OK.
        schema:
          $ref: '#/definitions/ReturnInfo'
        examples:
          returninfo: {"msg_code":"SUCCESS","msg":"文件移动成功","data":{"status": "运行的状态"}}
    :return:
    """
    ret_obj = ReturnInfo()
    jsons = request.json
    ruleid = jsons.get("ruleid")
    rule = Rule.query.filter_by(id=ruleid).first()
    if rule is None:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "规则id 不存在"
        return jsonify(ret_obj.todict())
    if rule.rule_type == 0:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "该条规则为单次提取无法废弃"
        return jsonify(ret_obj.todict())
    elif rule.rule_type == 1:
        pull_dir = current_app.config.get("PULL_ROUTINE")
    else:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "数据库　rule_type　错误"
        return jsonify(ret_obj.todict())

    rule.is_deprecated = 1
    try:
        db.session.add(rule)
        db.session.commit()
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "rule的is_deprecated状态 写入数据库失败"
        ret_obj.data = {"errmsg": str(e), "results": jsons}
        return jsonify(ret_obj.todict())

    pull_dir = pull_dir + "/" + str(rule.userid) + "_" + str(rule.id)
    web_hdfs_uri = current_app.config.get("WEB_HDFS")
    requests_session = requests.api
    kwargs = {}
    kwargs["user.name"] = "qhy"
    # 查看目录
    kwargs["op"] = "GETFILESTATUS"
    response = requests_session.request("get",
                                        "{}{}".format(web_hdfs_uri, pull_dir),
                                        params=kwargs,
                                        timeout=30,
                                        allow_redirects=False)
    if change_hdfs(response):
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "webhdfs 已切换,请重新访问"
        ret_obj.data = {}
        return jsonify(ret_obj.todict())
    if "RemoteException" in response.json():
        try:
            if response.json()["RemoteException"]["exception"] == "FileNotFoundException":
                weblog.info(g.weblog_id + "不存在目录,不需要刪除")
            else:
                ret_obj.msg_code = "FAILED"
                ret_obj.msg = "删除目录及里面的文件失败"
                ret_obj.data = response.json()
                return jsonify(ret_obj.todict())
        except:
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "删除目录及里面的文件失败"
            ret_obj.data = response.json()
            return jsonify(ret_obj.todict())
    else:
        kwargs["op"] = "DELETE"
        kwargs["recursive"] = True
        response = requests_session.request("delete",
                                            "{}{}".format(web_hdfs_uri, pull_dir),
                                            params=kwargs,
                                            timeout=30,
                                            allow_redirects=False)
        if "boolean" in response.json() and response.json()["boolean"]:
            weblog.info(g.weblog_id + ";存在目录,删除目录及里面的文件成功")
        else:
            weblog.info(g.weblog_id + ",删除目录及里面的文件失败")
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "删除目录及里面的文件失败"
            ret_obj.data = response.json()
            return jsonify(ret_obj.todict())

    ret_obj.msg_code = "SUCCESS"
    ret_obj.msg = "状态设置成功"
    return jsonify(ret_obj.todict())


@api.route("/pulldata/down_file", methods=['POST'])
# @verify_json_schema(down_file_schema)
@verify_token_auth(allowtokennull=False)
def down_file():
    """
    下载文件　该接口一次下载一个文件
    ---
    tags:
      - pulldata
    parameters:
      - name: token
        in: formData
        description: token值
        type: string
        required: true
      - name: ruleid
        in: formData
        description: 规则列表返回的id
        type: string
        required: true
      - name: filename
        in: formData
        description: 文件名
        type: string
        required: true
    consumes:
      - multipart/form-data
    produces:
      - application/octet-stream
    responses:
      200:
        description: 返回文件流
    :return:
    """
    ret_obj = ReturnInfo()
    ruleid = request.form['ruleid']
    filename = request.form['filename']
    rule = Rule.query.filter_by(id=ruleid).first()
    if rule is None:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "规则id 不存在"
        return jsonify(ret_obj.todict())
    if rule.rule_type == 0:
        pull_dir = current_app.config.get("PULL_ONCE")
    elif rule.rule_type == 1:
        pull_dir = current_app.config.get("PULL_ROUTINE")
    else:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "数据库　rule_type　错误"
        return jsonify(ret_obj.todict())
    pull_dir = pull_dir + "/" + str(rule.userid) + "_" + str(rule.id)
    web_hdfs_uri = current_app.config.get("WEB_HDFS")

    kwargs = {}
    kwargs["user.name"] = current_app.config.get("FILES_USER")
    kwargs["overwrite"] = False
    requests_session = requests.api

    response = ""
    have_file = False
    try:
        kwargs["op"] = "LISTSTATUS"
        response = requests_session.request("get",
                                            "{}{}".format(web_hdfs_uri, pull_dir),
                                            params=kwargs,
                                            timeout=30,
                                            allow_redirects=False)
        if change_hdfs(response):
            ret_obj.msg_code = "FAILED"
            ret_obj.msg = "webhdfs 已切换,请重新访问"
            ret_obj.data = {}
            return jsonify(ret_obj.todict())
        lists = []
        for item in response.json()["FileStatuses"]["FileStatus"]:
            if filename == item["pathSuffix"]:
                have_file = True
                break
    except Exception as e:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "查询{}目录错误".format("{}{}".format(web_hdfs_uri, pull_dir))
        ret_obj.data = {"errmsg": str(e), "result": response.json()}
        return jsonify(ret_obj.todict())
    if not have_file:
        ret_obj.msg_code = "FAILED"
        ret_obj.msg = "文件不在目录下"
        ret_obj.data = {"result": response.json()}
        return jsonify(ret_obj.todict())
    from pyhdfs import HdfsClient
    hosts = current_app.config.get("HOSTS")
    client = HdfsClient(hosts=hosts, user_name="qhy")
    content_type = 'application/octet-stream'
    filepath = "{}/{}".format(pull_dir, filename)

    def generate():
        with client.open(filepath, overwrite=True) as fsrc:
            length = 8 * 1024
            while 1:
                buf = fsrc.read(length)
                if not buf:
                    break
                yield buf

    response = Response(stream_with_context(generate()))
    response.headers['Content-Type'] = content_type
    response.headers['Content-Disposition'] = 'attachment; filename={}'.format(filename)
    return response
